In [1]:
###################################################################
# Built-in modules ################################################
###################################################################

import sys

sys.path.append("../../")

import os
import numpy as np
import imp
import cv2
import os
import matplotlib.pyplot as plt
import gc

import math
import time
import glob

###################################################################
# Custom modules ##################################################
###################################################################

from components.matchers.NumbaPatchMatcher import Wrapper as m
from components.utils.SimpleProgressBar import SimpleProgressBar
from components.utils import utils as u
from components.utils import plot_utils as plu
from components.utils.CSVWriter2 import Wrapper as csv
from components.utils.Metrix import Wrapper as me

###################################################################
# Dataset specific modules#########################################
###################################################################

from components.utils import middlebury_utils as mbu
In [2]:
ROOT_PATH = os.path.join("..", "..")
EXPERIMENT_TITLE = "ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba"

INIT_METHOD = "maclean"
DATASET = "middlebury"

DATASET_FOLDER = os.path.join(ROOT_PATH, "datasets", DATASET)
LOG_FOLDER = os.path.join(ROOT_PATH, "experiments", "logs")
CSV_FILEPATH = os.path.join(LOG_FOLDER, EXPERIMENT_TITLE+".csv")
IMG_RES = "450X375"
PREPROCESSING_METHOD = "None"
KERNEL_SIZE = 1
KERNEL_SPEC = "None"

SCENES = ["teddy", "cones"]
SIZE=""
YEAR= 2003
EXP_PARAMS = dict()

Loading the dataset and visualising the results

In [3]:
loaded_imgs_and_paths = list(mbu.get_images(DATASET_FOLDER, YEAR, scene) for scene in SCENES)

for im, path in loaded_imgs_and_paths:
    filenames = list(os.path.split(p)[-1] for p in path)
    plu.plot_images(im, filenames)
['im2.png', 'im6.png', 'disp2.png', 'teddy_nonocc.png']
..\..\datasets\middlebury\middlebury_2003\teddy\im2.png
..\..\datasets\middlebury\middlebury_2003\teddy\im6.png
..\..\datasets\middlebury\middlebury_2003\teddy\disp2.png
..\..\datasets\middlebury\middlebury_2003\teddy\teddy_nonocc.png
['im2.png', 'im6.png', 'disp2.png', 'cones_nonocc.png']
..\..\datasets\middlebury\middlebury_2003\cones\im2.png
..\..\datasets\middlebury\middlebury_2003\cones\im6.png
..\..\datasets\middlebury\middlebury_2003\cones\disp2.png
..\..\datasets\middlebury\middlebury_2003\cones\cones_nonocc.png

The addition of occluded areas to the pristine groundtruth images

In [4]:
###################################################################
# 2. Occluded areas are added to the groundtruths #################
###################################################################

nonocc_gts = list()
for imgs, paths in loaded_imgs_and_paths:
    temp = mbu.add_masks_to_raw_disparity(imgs[2], imgs[3])
    nonocc_gts.append(mbu.add_masks_to_raw_disparity(imgs[2], imgs[3]))
plu.plot_images(nonocc_gts, ["teddy_nonocc_gt", "cones_nonocc_gt"])

Benchmarking and evaluation

In [5]:
###################################################################
# 3.1 Initialising hyperparameters and matcher#####################
###################################################################

csv_logger = csv(CSV_FILEPATH, default_header=False)
csv_logger.set_header_function(csv_logger.get_header_v3)
csv_logger.write_csv_header()
csv_logger.set_line_function(csv.format_stereo_matching_results_v2)

MATCH = 60
GAP = -20
EGAP = -1


EXP_PARAMS = {"experiment_id":EXPERIMENT_TITLE, "match": MATCH,"gap":GAP,"egap":EGAP,\
                  "algo":str(m.__module__), "init_method":"default","dataset":DATASET,\
                  "preprocessing_method":"None","kernel_size":1,"kernel_spec":"None"}
Directory '..\..\experiments\logs' already exists.
In [6]:
###################################################################
# 3.2 Matching ####################################################
###################################################################
from skimage.morphology import disk, diamond, star, octagon
from skimage import filters

results = []
EXP_PARAMS["init_method"] = "maclean_et_al"
patches = dict(    
               #Gaussian filters
               gauss_3_3 = u.gaussian_kernel(3,3,1,1)*9,
               gauss_5_5 = u.gaussian_kernel(5,5,1,1) *25,
               gauss_5_3 = u.gaussian_kernel(5,3,1,1) *15, 
               gauss_7_3 = u.gaussian_kernel(7,3,1,1) *21,
               gauss_11_3 = u.gaussian_kernel(11,3,1,1) *33, 
               gauss_13_3 = u.gaussian_kernel(13,3,1,1) *39,
    
               #Sci-image morphological
               
               star_2=star(2),
               star_3=star(3), 
               octagon_1_3 = octagon(1,3),
               octagon_3_1 = octagon(3,1),
               diamond_2 = diamond(2), 
               diamond_3=diamond(3),
               disk_2=disk(2),
               disk_3=disk(3),
               disk_4=disk(4),
               
               #Farid filters
    
               hfarid = filters.edges.HFARID_WEIGHTS,
               vfarid = filters.edges.VFARID_WEIGHTS,
               hvfarid_div2 = (filters.edges.HFARID_WEIGHTS+filters.edges.VFARID_WEIGHTS)/2,
               hvfarid = (filters.edges.HFARID_WEIGHTS+filters.edges.VFARID_WEIGHTS),
               
               #Sobel filters
    
               hsobel = filters.edges.HSOBEL_WEIGHTS,
               vsobel = filters.edges.VSOBEL_WEIGHTS,
               hvsobel_div2 = (filters.edges.HSOBEL_WEIGHTS+filters.edges.VSOBEL_WEIGHTS)/2,
               hvsobel_div = (filters.edges.HSOBEL_WEIGHTS+filters.edges.VSOBEL_WEIGHTS),
               
               #Scharr filters
    
               hscharr = filters.edges.HSCHARR_WEIGHTS,
               vscharr = filters.edges.VSCHARR_WEIGHTS,
               hvscharr_div2 = (filters.edges.HSCHARR_WEIGHTS+filters.edges.VSCHARR_WEIGHTS)/2,
               hvscharr = (filters.edges.HSCHARR_WEIGHTS+filters.edges.VSCHARR_WEIGHTS),
               
               #Prewitt filters
    
               hprewitt = filters.edges.VPREWITT_WEIGHTS,
               vprewitt = filters.edges.VPREWITT_WEIGHTS,
               hvprewitt_div2 = (filters.edges.VPREWITT_WEIGHTS+filters.edges.HPREWITT_WEIGHTS)/2,
               hvprewitt = (filters.edges.VPREWITT_WEIGHTS+filters.edges.HPREWITT_WEIGHTS)
               )
In [9]:
matcher = m(MATCH, GAP, EGAP)
progress_bar = SimpleProgressBar.get_instance()

progress_counter = 1
steps_to_be_taken = len(SCENES)*len(patches)*4
print("Number of iterations: {0}".format(steps_to_be_taken))

SAVE_PATH  = os.path.join(ROOT_PATH, "experiments", "disparities", EXPERIMENT_TITLE)

saved_image_names = []
results = []
Number of iterations: 248
In [10]:
for match_value in range(10, 41, 10):
    for k,p in patches.items():
        for i, scene in enumerate(SCENES):
            # Parameters to be logged...

            EXP_PARAMS["scene"] = scene
            EXP_PARAMS["image_filename"] = IMAGE_FILENAME = loaded_imgs_and_paths[i][1][0]
            EXP_PARAMS["img_res"] = "{0}x{1}".format(loaded_imgs_and_paths[i][0][1].shape[1], loaded_imgs_and_paths[i][0][1].shape[0])  
            EXP_PARAMS["kernel_size"] = "{0}x{1}".format(p.shape[0], p.shape[1])
            EXP_PARAMS["kernel_spec"] = k

            EXP_PARAMS["match"] = MATCH = match_value
            matcher = m(MATCH, GAP, EGAP)


            occ = loaded_imgs_and_paths[i][0][3]
            gt = loaded_imgs_and_paths[i][0][2]

            matcher.set_images(loaded_imgs_and_paths[i][0][0].astype(np.float64),
                               loaded_imgs_and_paths[i][0][1].astype(np.float64))
            tic = time.time()

            matcher.configure_instance()
            matcher.set_filter(p)

            tic=time.time()

            x,raw_disp_map = matcher.test_pipeline()
            toc = time.time()

            EXP_PARAMS["runtime"] = toc-tic

            disp = raw_disp_map
            disp = 4*disp
            
            temp_path = u.save_disparity(SAVE_PATH, disp)
            saved_image_names.append(os.path.split(temp_path)[-1])
            EXP_PARAMS["image_filename"] = temp_path


            EXP_PARAMS["are_occlusions_errors"] = ARE_OCCLUSIONS_ERRORS = False

            EXP_PARAMS["bad1"], EXP_PARAMS["bad2"], EXP_PARAMS["bad4"], EXP_PARAMS["BAD8"], EXP_PARAMS["abs_error"], EXP_PARAMS["mse"], EXP_PARAMS["avg"], EXP_PARAMS["eucledian"] =\
            BAD1, BAD2, BAD4, BAD8, ABS_ERR, MSE, AVG, EUCLEDIAN = me.evaluate_over_all(disp, gt, occ, occlusions_counted_in_errors = ARE_OCCLUSIONS_ERRORS)

            csv_logger.append_new_sm_results(EXP_PARAMS, selected_keys = csv.get_header_v3())

            EXP_PARAMS["are_occlusions_errors"] = True
            EXP_PARAMS["bad1"], EXP_PARAMS["bad2"], EXP_PARAMS["bad4"], EXP_PARAMS["BAD8"], EXP_PARAMS["abs_error"], EXP_PARAMS["mse"], EXP_PARAMS["avg"], EXP_PARAMS["eucledian"] =\
            BAD1, BAD2, BAD4, BAD8, ABS_ERR, MSE, AVG, EUCLEDIAN = me.evaluate_over_all(disp, gt, gt, occlusions_counted_in_errors = ARE_OCCLUSIONS_ERRORS)

            csv_logger.append_new_sm_results(EXP_PARAMS, selected_keys = csv.get_header_v3())
            progress_bar.progress_bar(progress_counter, steps_to_be_taken, header= "Experiment on patch sizes in progress: ", progress_bar_steps=40)
            progress_counter+=1
            results.append(disp)
Experiment on patch sizes in progress:  [>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>] 100.00 % completed: 248 / 248 steps
In [11]:
import pandas as pd
experiment_log = pd.read_csv(CSV_FILEPATH)
experiment_log.sort_values(by="abs_error", inplace=True) 
experiment_log.head(30)
Out[11]:
experiment_id match gap egap algo init_method dataset scene image_filename img_res ... are_occlusions_errors abs_error mse avg eucledian bad1 bad2 bad4 bad8 runtime
124 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 976791.0 437.297783 6.615539 8035.387670 0.588069 0.399239 0.248715 0.128811 28.180546
286 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1021402.0 692.589192 7.096716 9984.066907 0.393362 0.207579 0.113954 0.088671 59.418660
290 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1021402.0 692.589192 7.096716 9984.066907 0.393362 0.207579 0.113954 0.088671 60.826976
282 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1021546.0 692.745272 7.097717 9985.191836 0.393362 0.207579 0.113954 0.088678 41.466947
278 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1022536.0 692.647666 7.104595 9984.488369 0.393633 0.207815 0.114163 0.088942 40.675168
194 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1023788.0 605.394995 7.113294 9334.456599 0.445402 0.270590 0.154767 0.110022 60.794486
186 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1023788.0 605.394995 7.113294 9334.456599 0.445402 0.270590 0.154767 0.110022 59.886121
276 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1024225.0 445.157019 6.936797 8107.273216 0.556678 0.357282 0.206033 0.111594 27.790208
68 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1028489.0 423.631286 6.965676 7908.829433 0.557876 0.358237 0.216409 0.121347 41.065278
60 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1028489.0 423.631286 6.965676 7908.829433 0.557876 0.358237 0.216409 0.121347 70.255300
152 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1030843.0 419.834657 6.981619 7873.309787 0.589078 0.396340 0.225755 0.110226 34.590581
156 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1034153.0 422.453156 7.004037 7897.824447 0.589024 0.395981 0.225274 0.110158 55.164268
160 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1034189.0 422.457870 7.004280 7897.868510 0.589024 0.395981 0.225295 0.110179 63.222056
164 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1034189.0 422.457870 7.004280 7897.868510 0.589024 0.395981 0.225295 0.110179 64.782939
268 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1039809.0 454.937813 7.042343 8195.854013 0.557267 0.357553 0.206020 0.112238 24.943267
144 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1043225.0 429.554063 7.065479 7963.924096 0.592207 0.401392 0.228837 0.111012 31.577710
148 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1044795.0 433.773310 7.076112 8002.940897 0.586254 0.392351 0.223317 0.111201 53.219400
274 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1044984.0 710.481442 7.260564 10112.208067 0.396885 0.211511 0.117185 0.090894 34.156969
108 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury teddy ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1045177.0 523.693514 7.078699 8793.399286 0.597111 0.411802 0.260005 0.132962 24.671909
270 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 30 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1045942.0 705.021136 7.267221 10073.275138 0.400108 0.215548 0.121014 0.092513 25.794790
178 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1047390.0 615.245390 7.277281 9410.090754 0.450079 0.276065 0.157831 0.112586 69.836569
190 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 20 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1047390.0 615.245390 7.277281 9410.090754 0.450079 0.276065 0.157831 0.112586 62.709408
126 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1048498.0 597.573142 7.284980 9273.958809 0.464225 0.294978 0.171213 0.120743 28.917746
410 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 40 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1049996.0 689.720940 7.295388 9963.371718 0.408543 0.225790 0.127322 0.096758 54.795754
406 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 40 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1049996.0 689.720940 7.295388 9963.371718 0.408543 0.225790 0.127322 0.096758 48.677523
414 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 40 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1049996.0 689.720940 7.295388 9963.371718 0.408543 0.225790 0.127322 0.096758 48.689626
402 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 40 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1050854.0 689.160207 7.301349 9959.320860 0.408745 0.226054 0.127830 0.096925 34.626972
110 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1052532.0 575.953865 7.313008 9104.654634 0.474779 0.308339 0.181371 0.126926 24.309743
62 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1055134.0 686.931395 7.331087 9943.203106 0.423495 0.242882 0.133520 0.095070 46.225175
70 ALG_005_EXP_002-PatchMatch-MacLean_et_al-Numba 10 -20 -1 components.matchers.NumbaPatchMatcher maclean_et_al middlebury cones ..\..\experiments\disparities\ALG_005_EXP_002-... 450x375 ... False 1055134.0 686.931395 7.331087 9943.203106 0.423495 0.242882 0.133520 0.095070 39.505482

30 rows × 23 columns

In [12]:
plu.plot_images(results, saved_image_names, ncols=2)